{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AlexNet\n",
    "\n",
    "AlexNet 的原始论文 https://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf\n",
    "\n",
    "\n",
    "AlexNet结构图:\n",
    "\n",
    "\n",
    "![AlexNet结构图](../images/AlexNet.jpg)\n",
    "\n",
    "\n",
    "\n",
    "AlexNet取得效果好还有几个原因：\n",
    "\n",
    "1、使用ReLU作为激活函数\n",
    "\n",
    "2、局部正则化\n",
    "\n",
    "3、Dropout\n",
    "\n",
    "4、Data Augmentation(数据增强)\n",
    "\n",
    "接下来照着这个图，用PyTorch写一遍模型。当然，就不跑了。\n",
    "\n",
    "*附上一张NG在 coursera 上授课的AlexNet模型图*\n",
    "\n",
    "\n",
    "![AlexNet结构图](../images/AlexNet2.jpg)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "import torch.nn.functional as F\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "class AlextNet(nn.Module):\n",
    "    def __init__(self, in_channel, n_class):\n",
    "        super(AlextNet, self).__init__()\n",
    "        # 第一阶段\n",
    "        self.conv1 = nn.Sequential(\n",
    "            nn.Conv2d(in_channel, 96, kernel_size=11, stride=4),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(kernel_size=3, stride=2)\n",
    "        )\n",
    "        # 第二阶段\n",
    "        self.conv2 = nn.Sequential(\n",
    "            nn.Conv2d(96, 256, kernel_size=5, padding=2),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(kernel_size=3, stride=2)\n",
    "        )\n",
    "        # 第三阶段\n",
    "        self.conv3 = nn.Sequential(\n",
    "            nn.Conv2d(256, 384, kernel_size=3, padding=1),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv2d(384, 384, kernel_size=3, padding=1),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv2d(384, 256, kernel_size=3, padding=1),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(kernel_size=3, stride=2)\n",
    "        )\n",
    "        # 第四阶段 全连接层\n",
    "        self.fc = nn.Sequential(\n",
    "            nn.Linear(1*1*256, 4096),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(4096, 4096),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(4096, n_class) # AlexNet上面是1000 ...如果测试的话用MNIST则可以使用10\n",
    "        )\n",
    "    # 向前传播\n",
    "    def forward(self, x):\n",
    "        con1_x = self.conv1(x)\n",
    "        con2_x = self.conv2(con1_x)\n",
    "        con3_x = self.conv3(con2_x)\n",
    "        lin_x = con3_x.view(con3_x.size(0), -1)\n",
    "        y_hat = self.fc(lin_x)\n",
    "        return y_hat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AlextNet (\n",
      "  (conv1): Sequential (\n",
      "    (0): Conv2d(3, 96, kernel_size=(11, 11), stride=(4, 4))\n",
      "    (1): ReLU (inplace)\n",
      "    (2): MaxPool2d (size=(3, 3), stride=(2, 2), dilation=(1, 1))\n",
      "  )\n",
      "  (conv2): Sequential (\n",
      "    (0): Conv2d(96, 256, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))\n",
      "    (1): ReLU (inplace)\n",
      "    (2): MaxPool2d (size=(3, 3), stride=(2, 2), dilation=(1, 1))\n",
      "  )\n",
      "  (conv3): Sequential (\n",
      "    (0): Conv2d(256, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (1): ReLU (inplace)\n",
      "    (2): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (3): ReLU (inplace)\n",
      "    (4): Conv2d(384, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (5): ReLU (inplace)\n",
      "    (6): MaxPool2d (size=(3, 3), stride=(2, 2), dilation=(1, 1))\n",
      "  )\n",
      "  (fc): Sequential (\n",
      "    (0): Linear (9216 -> 4096)\n",
      "    (1): ReLU (inplace)\n",
      "    (2): Dropout (p = 0.5)\n",
      "    (3): Linear (4096 -> 4096)\n",
      "    (4): ReLU (inplace)\n",
      "    (5): Dropout (p = 0.5)\n",
      "    (6): Linear (4096 -> 10)\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "alex = AlextNet(3, 10)\n",
    "print(alex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 图像预处理，因为Alex 是使用 227 * 227 大小的图片，但是 CIFAR10 只有 32 * 32 ,经过测试， 227 * 227 的效果不好。\n",
    "# 所以这里， 我们将图像放大到 96*96\n",
    "transform = transforms.Compose([\n",
    "    transforms.Resize(96),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n",
    "])\n",
    "\n",
    "# 超参数\n",
    "DOWNLOAD = True\n",
    "BATCH_SIZE = 256\n",
    "EPOCH = 5\n",
    "learning_rate = 0.001\n",
    "\n",
    "# 是否使用GPU\n",
    "use_gpu = True\n",
    "\n",
    "# CIFAR10 dataset\n",
    "train_dataset = torchvision.datasets.CIFAR10(root='./data/', train=True, transform=transform, download=DOWNLOAD)\n",
    "\n",
    "test_dataset = torchvision.datasets.CIFAR10(root='./data/', train=False, transform=transform)\n",
    "\n",
    "# Data Loader\n",
    "train_loader = DataLoader(dataset=train_dataset,\n",
    "                          batch_size=BATCH_SIZE,\n",
    "                          shuffle=True)\n",
    "\n",
    "test_loader = DataLoader(dataset=test_dataset,\n",
    "                         batch_size=BATCH_SIZE,\n",
    "                         shuffle=False)\n",
    "\n",
    "# 定义模型\n",
    "alex = AlextNet(3, 10)\n",
    "if use_gpu:\n",
    "    alex = alex.cuda()\n",
    "\n",
    "# loss and optimizer\n",
    "\n",
    "loss_fn = nn.CrossEntropyLoss()\n",
    "\n",
    "optimizer = torch.optim.Adam(alex.parameters(), lr=learning_rate)\n",
    "\n",
    "# Training\n",
    "alex.train()\n",
    "\n",
    "for epoch in range(EPOCH):\n",
    "    total = 0\n",
    "    correct = 0\n",
    "    for i, (images, labels) in enumerate(train_loader):\n",
    "        images = Variable(images)\n",
    "        labels = Variable(labels)\n",
    "\n",
    "        if use_gpu:\n",
    "            images = images.cuda()\n",
    "            labels = labels.cuda()\n",
    "        # forward + backward + optimize\n",
    "        optimizer.zero_grad()\n",
    "        y_pred = alex(images)\n",
    "\n",
    "        loss = loss_fn(y_pred, labels)\n",
    "\n",
    "        loss.backward()\n",
    "\n",
    "        optimizer.step()\n",
    "\n",
    "        if (i + 1) % 100 == 0:\n",
    "            print(\"Epoch [%d/%d], Iter [%d/%d] Loss: %.4f\" % (epoch + 1, EPOCH, i + 1, 100, loss.data[0]))\n",
    "\n",
    "        # 计算训练精确度\n",
    "        _, predicted = torch.max(y_pred.data, 1)\n",
    "        total += labels.size(0)\n",
    "        correct += (predicted == labels.data).sum()\n",
    "    print('Accuracy of the model on the train images: %d %%' % (100 * correct / total))\n",
    "\n",
    "    # Decaying Learning Rate\n",
    "    if (epoch+1) % 2 == 0:\n",
    "        learning_rate /= 3\n",
    "        optimizer = torch.optim.Adam(alex.parameters(), lr=learning_rate)\n",
    "\n",
    "\n",
    "# Test\n",
    "alex.eval()\n",
    "\n",
    "correct = 0\n",
    "total = 0\n",
    "\n",
    "for images, labels in test_loader:\n",
    "    images = Variable(images)\n",
    "    labels = Variable(labels)\n",
    "    if use_gpu:\n",
    "        images = images.cuda()\n",
    "        labels = labels.cuda()\n",
    "\n",
    "    y_pred = alex(images)\n",
    "    _, predicted = torch.max(y_pred.data, 1)\n",
    "    total += labels.size(0)\n",
    "    temp = (predicted == labels.data).sum()\n",
    "    correct += temp\n",
    "\n",
    "print('Accuracy of the model on the test images: %d %%' % (100 * correct / total))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "跑了5次之后的结果。用 GPU 跑，大概1min多\n",
    "\n",
    "> Files already downloaded and verified\n",
    "\n",
    "> Epoch [1/5], Iter [100/100] Loss: 1.7411\n",
    "\n",
    "> Accuracy of the model on the train images: 28 %\n",
    "\n",
    "> Epoch [2/5], Iter [100/100] Loss: 1.4252\n",
    "\n",
    "> Accuracy of the model on the train images: 46 %\n",
    "\n",
    "> Epoch [3/5], Iter [100/100] Loss: 1.1083\n",
    "\n",
    "> Accuracy of the model on the train images: 57 %\n",
    "\n",
    "> Epoch [4/5], Iter [100/100] Loss: 0.9585\n",
    "\n",
    "> Accuracy of the model on the train images: 61 %\n",
    "\n",
    "> Epoch [5/5], Iter [100/100] Loss: 0.9199\n",
    "\n",
    "> Accuracy of the model on the train images: 66 %\n",
    "\n",
    "> Accuracy of the model on the test images: 65 %\n"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
